{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jRDuJsGCgxCO"
      },
      "source": [
        "# HW3 Image Classification\n",
        "## We strongly recommend that you run with [Kaggle](https://www.kaggle.com/t/86ca241732c04da99aca6490080bae73) for this homework\n",
        "\n",
        "If you have any questions, please contact the TAs via TA hours, NTU COOL, or email to mlta-2023-spring@googlegroups.com"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K9Vxu1iQwdIE"
      },
      "source": [
        "# Check GPU Type"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KKjf6sZcwb_A",
        "outputId": "605652be-d5bf-48a2-8218-57b33b6f7790"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mon Mar 13 14:55:35 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.85.12    Driver Version: 525.85.12    CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   66C    P0    31W /  70W |    625MiB / 15360MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EVgrPb3HhJUT"
      },
      "source": [
        "# Get Data\n",
        "Notes: if the links are dead, you can download the data directly from Kaggle and upload it to the workspace, or you can use the Kaggle API to directly download the data into colab.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EAO6dg9eVaU_",
        "outputId": "5a20fe0a-ab0f-40e8-82d7-5cb1dea91d92"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: gdown in /usr/local/lib/python3.9/dist-packages (4.6.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from gdown) (3.9.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.9/dist-packages (from gdown) (1.15.0)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.9/dist-packages (from gdown) (4.6.3)\n",
            "Requirement already satisfied: requests[socks] in /usr/local/lib/python3.9/dist-packages (from gdown) (2.25.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.9/dist-packages (from gdown) (4.65.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests[socks]->gdown) (1.26.14)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests[socks]->gdown) (2022.12.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests[socks]->gdown) (2.10)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.9/dist-packages (from requests[socks]->gdown) (4.0.0)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.9/dist-packages (from requests[socks]->gdown) (1.7.1)\n",
            "/usr/local/lib/python3.9/dist-packages/gdown/cli.py:121: FutureWarning: Option `--id` was deprecated in version 4.3.1 and will be removed in 5.0. You don't need to pass it anymore to use a file ID.\n",
            "  warnings.warn(\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1tbGNwk1yGoCBdu4Gi_Cia7EJ9OhubYD9\n",
            "To: /content/food11.zip\n",
            "100% 1.16G/1.16G [00:09<00:00, 126MB/s] \n"
          ]
        }
      ],
      "source": [
        "# Download Link\n",
        "# Link 1 (Dropbox): https://www.dropbox.com/s/up5q1gthsz3v0dq/food-11.zip?dl=0\n",
        "# Link 2 (Google Drive): https://drive.google.com/file/d/1tbGNwk1yGoCBdu4Gi_Cia7EJ9OhubYD9/view?usp=share_link\n",
        "# Link 3: Kaggle Competition.\n",
        "\n",
        "# (1) dropbox link\n",
        "# !wget -O food11.zip https://www.dropbox.com/s/up5q1gthsz3v0dq/food-11.zip?dl=0\n",
        "\n",
        "# (2) google drive link\n",
        "!pip install gdown --upgrade\n",
        "!gdown --id '1tbGNwk1yGoCBdu4Gi_Cia7EJ9OhubYD9' --output food11.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "HEsBm1lkhGmk",
        "outputId": "96b56b94-7aee-4180-b615-61a1cff7c7e5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Archive:  food11.zip\n",
            "replace valid/9_2898.jpg? [y]es, [n]o, [A]ll, [N]one, [r]ename: "
          ]
        }
      ],
      "source": [
        "! unzip food11.zip"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n5ceUnRihL-f"
      },
      "source": [
        "# Import Packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "ay3WkYnHVaVE"
      },
      "outputs": [],
      "source": [
        "_exp_name = \"sample\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "CwOGtRWHVaVF"
      },
      "outputs": [],
      "source": [
        "# Import necessary packages.\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import os\n",
        "import torch.nn as nn\n",
        "import torchvision.transforms as transforms\n",
        "from PIL import Image\n",
        "# \"ConcatDataset\" and \"Subset\" are possibly useful when doing semi-supervised learning.\n",
        "from torch.utils.data import ConcatDataset, DataLoader, Subset, Dataset\n",
        "from torchvision.datasets import DatasetFolder, VisionDataset\n",
        "# This is for the progress bar.\n",
        "from tqdm.auto import tqdm\n",
        "import random"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "8kJm9GekVaVH"
      },
      "outputs": [],
      "source": [
        "myseed = 6666  # set a random seed for reproducibility\n",
        "torch.backends.cudnn.deterministic = True\n",
        "torch.backends.cudnn.benchmark = False\n",
        "np.random.seed(myseed)\n",
        "torch.manual_seed(myseed)\n",
        "if torch.cuda.is_available():\n",
        "    torch.cuda.manual_seed_all(myseed)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d9MVtgbSVaVH"
      },
      "source": [
        "# Transforms\n",
        "Torchvision provides lots of useful utilities for image preprocessing, data *wrapping* as well as data augmentation.\n",
        "\n",
        "Please refer to PyTorch official website for details about different transforms."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "jvI3Xmq4VaVJ"
      },
      "outputs": [],
      "source": [
        "# Normally, We don't need augmentations in testing and validation.\n",
        "# All we need here is to resize the PIL image and transform it into Tensor.\n",
        "test_tfm = transforms.Compose([\n",
        "    transforms.Resize((128, 128)),\n",
        "    transforms.ToTensor(),\n",
        "])\n",
        "\n",
        "# However, it is also possible to use augmentation in the testing phase.\n",
        "# You may use train_tfm to produce a variety of images and then test using ensemble methods\n",
        "train_tfm = transforms.Compose([\n",
        "    # Resize the image into a fixed shape (height = width = 128)\n",
        "    transforms.Resize((128, 128)),\n",
        "    # You may add some transforms here.\n",
        "\n",
        "    # ToTensor() should be the last one of the transforms.\n",
        "    transforms.ToTensor(),\n",
        "])\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D0ivMf-jVaVK"
      },
      "source": [
        "# Datasets\n",
        "The data is labelled by the name, so we load images and label while calling '__getitem__'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "xBdtPhKwVaVL"
      },
      "outputs": [],
      "source": [
        "class FoodDataset(Dataset):\n",
        "\n",
        "    def __init__(self,path,tfm=test_tfm,files = None):\n",
        "        super(FoodDataset).__init__()\n",
        "        self.path = path\n",
        "        self.files = sorted([os.path.join(path,x) for x in os.listdir(path) if x.endswith(\".jpg\")])\n",
        "        if files != None:\n",
        "            self.files = files\n",
        "\n",
        "        self.transform = tfm\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.files)\n",
        "\n",
        "    def __getitem__(self,idx):\n",
        "        fname = self.files[idx]\n",
        "        im = Image.open(fname)\n",
        "        im = self.transform(im)\n",
        "\n",
        "        try:\n",
        "            label = int(fname.split(\"/\")[-1].split(\"_\")[0])\n",
        "        except:\n",
        "            label = -1 # test has no label\n",
        "\n",
        "        return im,label"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZPFkDwug61PZ"
      },
      "source": [
        "# Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "b_kDECOJVaVL"
      },
      "outputs": [],
      "source": [
        "class Classifier(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Classifier, self).__init__()\n",
        "        # torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding)\n",
        "        # torch.nn.MaxPool2d(kernel_size, stride, padding)\n",
        "        # input 維度 [3, 128, 128]\n",
        "        self.cnn = nn.Sequential(\n",
        "            nn.Conv2d(3, 64, 3, 1, 1),  # [64, 128, 128]\n",
        "            nn.BatchNorm2d(64),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(2, 2, 0),      # [64, 64, 64]\n",
        "\n",
        "            nn.Conv2d(64, 128, 3, 1, 1), # [128, 64, 64]\n",
        "            nn.BatchNorm2d(128),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(2, 2, 0),      # [128, 32, 32]\n",
        "\n",
        "            nn.Conv2d(128, 256, 3, 1, 1), # [256, 32, 32]\n",
        "            nn.BatchNorm2d(256),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(2, 2, 0),      # [256, 16, 16]\n",
        "\n",
        "            nn.Conv2d(256, 512, 3, 1, 1), # [512, 16, 16]\n",
        "            nn.BatchNorm2d(512),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(2, 2, 0),       # [512, 8, 8]\n",
        "\n",
        "            nn.Conv2d(512, 512, 3, 1, 1), # [512, 8, 8]\n",
        "            nn.BatchNorm2d(512),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(2, 2, 0),       # [512, 4, 4]\n",
        "        )\n",
        "        self.fc = nn.Sequential(\n",
        "            nn.Linear(512*4*4, 1024),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(1024, 512),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(512, 11)\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = self.cnn(x)\n",
        "        out = out.view(out.size()[0], -1)\n",
        "        return self.fc(out)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xgnIOaID687b"
      },
      "source": [
        "# Configurations"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "2_OeWtstVaVO"
      },
      "outputs": [],
      "source": [
        "# \"cuda\" only when GPUs are available.\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "\n",
        "# Initialize a model, and put it on the device specified.\n",
        "model = Classifier().to(device)\n",
        "\n",
        "# The number of batch size.\n",
        "batch_size = 64\n",
        "\n",
        "# The number of training epochs.\n",
        "n_epochs = 8\n",
        "\n",
        "# If no improvement in 'patience' epochs, early stop.\n",
        "patience = 5\n",
        "\n",
        "# For the classification task, we use cross-entropy as the measurement of performance.\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "# Initialize optimizer, you may fine-tune some hyperparameters such as learning rate on your own.\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=0.0003, weight_decay=1e-5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zPGwvvPK7F7u"
      },
      "source": [
        "# Dataloader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "zvZXRH2phItl"
      },
      "outputs": [],
      "source": [
        "# Construct train and valid datasets.\n",
        "# The argument \"loader\" tells how torchvision reads the data.\n",
        "train_set = FoodDataset(\"./train\", tfm=train_tfm)\n",
        "train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0, pin_memory=True)\n",
        "valid_set = FoodDataset(\"./valid\", tfm=test_tfm)\n",
        "valid_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=True, num_workers=0, pin_memory=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UPQ0j1c17BG1"
      },
      "source": [
        "# Start Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zbVkfIFhVaVO"
      },
      "outputs": [],
      "source": [
        "# Initialize trackers, these are not parameters and should not be changed\n",
        "stale = 0\n",
        "best_acc = 0\n",
        "\n",
        "for epoch in range(n_epochs):\n",
        "\n",
        "    # ---------- Training ----------\n",
        "    # Make sure the model is in train mode before training.\n",
        "    model.train()\n",
        "\n",
        "    # These are used to record information in training.\n",
        "    train_loss = []\n",
        "    train_accs = []\n",
        "\n",
        "    for batch in tqdm(train_loader):\n",
        "\n",
        "        # A batch consists of image data and corresponding labels.\n",
        "        imgs, labels = batch\n",
        "        #imgs = imgs.half()\n",
        "        #print(imgs.shape,labels.shape)\n",
        "\n",
        "        # Forward the data. (Make sure data and model are on the same device.)\n",
        "        logits = model(imgs.to(device))\n",
        "\n",
        "        # Calculate the cross-entropy loss.\n",
        "        # We don't need to apply softmax before computing cross-entropy as it is done automatically.\n",
        "        loss = criterion(logits, labels.to(device))\n",
        "\n",
        "        # Gradients stored in the parameters in the previous step should be cleared out first.\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # Compute the gradients for parameters.\n",
        "        loss.backward()\n",
        "\n",
        "        # Clip the gradient norms for stable training.\n",
        "        grad_norm = nn.utils.clip_grad_norm_(model.parameters(), max_norm=10)\n",
        "\n",
        "        # Update the parameters with computed gradients.\n",
        "        optimizer.step()\n",
        "\n",
        "        # Compute the accuracy for current batch.\n",
        "        acc = (logits.argmax(dim=-1) == labels.to(device)).float().mean()\n",
        "\n",
        "        # Record the loss and accuracy.\n",
        "        train_loss.append(loss.item())\n",
        "        train_accs.append(acc)\n",
        "\n",
        "    train_loss = sum(train_loss) / len(train_loss)\n",
        "    train_acc = sum(train_accs) / len(train_accs)\n",
        "\n",
        "    # Print the information.\n",
        "    print(f\"[ Train | {epoch + 1:03d}/{n_epochs:03d} ] loss = {train_loss:.5f}, acc = {train_acc:.5f}\")\n",
        "\n",
        "    # ---------- Validation ----------\n",
        "    # Make sure the model is in eval mode so that some modules like dropout are disabled and work normally.\n",
        "    model.eval()\n",
        "\n",
        "    # These are used to record information in validation.\n",
        "    valid_loss = []\n",
        "    valid_accs = []\n",
        "\n",
        "    # Iterate the validation set by batches.\n",
        "    for batch in tqdm(valid_loader):\n",
        "\n",
        "        # A batch consists of image data and corresponding labels.\n",
        "        imgs, labels = batch\n",
        "        #imgs = imgs.half()\n",
        "\n",
        "        # We don't need gradient in validation.\n",
        "        # Using torch.no_grad() accelerates the forward process.\n",
        "        with torch.no_grad():\n",
        "            logits = model(imgs.to(device))\n",
        "\n",
        "        # We can still compute the loss (but not the gradient).\n",
        "        loss = criterion(logits, labels.to(device))\n",
        "\n",
        "        # Compute the accuracy for current batch.\n",
        "        acc = (logits.argmax(dim=-1) == labels.to(device)).float().mean()\n",
        "\n",
        "        # Record the loss and accuracy.\n",
        "        valid_loss.append(loss.item())\n",
        "        valid_accs.append(acc)\n",
        "        #break\n",
        "\n",
        "    # The average loss and accuracy for entire validation set is the average of the recorded values.\n",
        "    valid_loss = sum(valid_loss) / len(valid_loss)\n",
        "    valid_acc = sum(valid_accs) / len(valid_accs)\n",
        "\n",
        "    # Print the information.\n",
        "    print(f\"[ Valid | {epoch + 1:03d}/{n_epochs:03d} ] loss = {valid_loss:.5f}, acc = {valid_acc:.5f}\")\n",
        "\n",
        "\n",
        "    # update logs\n",
        "    if valid_acc > best_acc:\n",
        "        with open(f\"./{_exp_name}_log.txt\",\"a\"):\n",
        "            print(f\"[ Valid | {epoch + 1:03d}/{n_epochs:03d} ] loss = {valid_loss:.5f}, acc = {valid_acc:.5f} -> best\")\n",
        "    else:\n",
        "        with open(f\"./{_exp_name}_log.txt\",\"a\"):\n",
        "            print(f\"[ Valid | {epoch + 1:03d}/{n_epochs:03d} ] loss = {valid_loss:.5f}, acc = {valid_acc:.5f}\")\n",
        "\n",
        "\n",
        "    # save models\n",
        "    if valid_acc > best_acc:\n",
        "        print(f\"Best model found at epoch {epoch}, saving model\")\n",
        "        torch.save(model.state_dict(), f\"{_exp_name}_best.ckpt\") # only save best to prevent output memory exceed error\n",
        "        best_acc = valid_acc\n",
        "        stale = 0\n",
        "    else:\n",
        "        stale += 1\n",
        "        if stale > patience:\n",
        "            print(f\"No improvment {patience} consecutive epochs, early stopping\")\n",
        "            break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rw2y9Uhw7wr2"
      },
      "source": [
        "# Dataloader for test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B9QNdHIXVaVP"
      },
      "outputs": [],
      "source": [
        "# Construct test datasets.\n",
        "# The argument \"loader\" tells how torchvision reads the data.\n",
        "test_set = FoodDataset(\"./test\", tfm=test_tfm)\n",
        "test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=0, pin_memory=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G31uyjpvVaVP"
      },
      "source": [
        "# Testing and generate prediction CSV"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bpLtxx5FVaVP"
      },
      "outputs": [],
      "source": [
        "model_best = Classifier().to(device)\n",
        "model_best.load_state_dict(torch.load(f\"{_exp_name}_best.ckpt\"))\n",
        "model_best.eval()\n",
        "prediction = []\n",
        "with torch.no_grad():\n",
        "    for data,_ in tqdm(test_loader):\n",
        "        test_pred = model_best(data.to(device))\n",
        "        test_label = np.argmax(test_pred.cpu().data.numpy(), axis=1)\n",
        "        prediction += test_label.squeeze().tolist()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fKupB3VUVaVQ"
      },
      "outputs": [],
      "source": [
        "# create test csv\n",
        "def pad4(i):\n",
        "    return \"0\"*(4-len(str(i)))+str(i)\n",
        "df = pd.DataFrame()\n",
        "df[\"Id\"] = [pad4(i) for i in range(len(test_set))]\n",
        "df[\"Category\"] = prediction\n",
        "df.to_csv(\"submission.csv\",index = False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ivk0hrE-V8Cu"
      },
      "source": [
        "# Q1. Augmentation Implementation\n",
        "## Implement augmentation by finishing train_tfm in the code with image size of your choice.\n",
        "## Directly copy the following block and paste it on GradeScope after you finish the code\n",
        "### Your train_tfm must be capable of producing 5+ different results when given an identical image multiple times.\n",
        "### Your  train_tfm in the report can be different from train_tfm in your training code.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GSfKNo42WjKm"
      },
      "outputs": [],
      "source": [
        "train_tfm = transforms.Compose([\n",
        "    # Resize the image into a fixed shape (height = width = 128)\n",
        "    transforms.Resize((128, 128)),\n",
        "    # You can add some transforms here.\n",
        "    transforms.ToTensor(),\n",
        "])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3HemRgZ6WwRM"
      },
      "source": [
        "# Q2. Visual Representations Implementation\n",
        "## Visualize the learned visual representations of the CNN model on the validation set by implementing t-SNE (t-distributed Stochastic Neighbor Embedding) on the output of both top & mid layers (You need to submit 2 images).\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iXd_SZnB2Wg8"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "from sklearn.manifold import TSNE\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm import tqdm\n",
        "import matplotlib.cm as cm\n",
        "import torch.nn as nn\n",
        "\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "\n",
        "# Load the trained model\n",
        "model = Classifier().to(device)\n",
        "state_dict = torch.load(f\"{_exp_name}_best.ckpt\")\n",
        "model.load_state_dict(state_dict)\n",
        "model.eval()\n",
        "\n",
        "print(model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QcBKUNfc3BeL"
      },
      "outputs": [],
      "source": [
        "# Load the vaildation set defined by TA\n",
        "valid_set = FoodDataset(\"./valid\", tfm=test_tfm)\n",
        "valid_loader = DataLoader(valid_set, batch_size=64, shuffle=False, num_workers=0, pin_memory=True)\n",
        "\n",
        "# Extract the representations for the specific layer of model\n",
        "index = ... # You should find out the index of layer which is defined as \"top\" or 'mid' layer of your model.\n",
        "features = []\n",
        "labels = []\n",
        "for batch in tqdm(valid_loader):\n",
        "    imgs, lbls = batch\n",
        "    with torch.no_grad():\n",
        "        logits = model.cnn[:index](imgs.to(device))\n",
        "        logits = logits.view(logits.size()[0], -1)\n",
        "    labels.extend(lbls.cpu().numpy())\n",
        "    logits = np.squeeze(logits.cpu().numpy())\n",
        "    features.extend(logits)\n",
        "\n",
        "features = np.array(features)\n",
        "colors_per_class = cm.rainbow(np.linspace(0, 1, 11))\n",
        "\n",
        "# Apply t-SNE to the features\n",
        "features_tsne = TSNE(n_components=2, init='pca', random_state=42).fit_transform(features)\n",
        "\n",
        "# Plot the t-SNE visualization\n",
        "plt.figure(figsize=(10, 8))\n",
        "for label in np.unique(labels):\n",
        "    plt.scatter(features_tsne[labels == label, 0], features_tsne[labels == label, 1], label=label, s=5)\n",
        "plt.legend()\n",
        "plt.show()\n",
        "\n",
        "plt.figure(figsize=(10, 8))\n",
        "labels = [0]\n",
        "for label in np.unique(labels):\n",
        "    plt.scatter(features_tsne[labels == label, 0], features_tsne[labels == label, 1], label=label, s=5)\n",
        "plt.legend()\n",
        "plt.show()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}